#!/usr/bin/python3
# author:szw
# cl
# 2023年06月29日
# Email:1259577135@qq.com
# desc: pinterest 网站采集
import time
import shutil
import fast_http
from gloab_key_words import key_words
import json
import  pinterest_util
import requests
from utils import MongoDBUtil, get_html, mongo_manager
import os
MongoDBUtil = MongoDBUtil()
pinterest_images= mongo_manager("pinterest_images", db="car_images")
proxies={'http':'127.0.0.1:15732',
         'https':'127.0.0.1:15732'}



def insert_mon_data(tablename,insert_data):
    try:
        MongoDBUtil.insert_one(tablename, insert_data)
    except Exception as e:
        print(e)



class pinterest_crawler(fast_http.Base_Model):

    def crawler(self, key_word):
        """
        采集一个关键字
        :param key_word:
        :return:
        """
        print(f"开始采集关键字 {key_word}")
        path = f'pinterest/{key_word}/'
        if os.path.exists(path):
            pass
        else:
            os.mkdir(path)
        bookmarks=""
        # bookmarks = "Y2JVSG81V2sxcmNHRlpWM1J5VFVad1dHTjZSazlpUlZwNFdWVmFRMVl3TVZkV2FsSlhUVlp3VkZaWGN6RlNhelZaVW14T1YxSnNjR2hYVjNoV1RWZE9WMVZZYUZaaVJUVnZWRlZTYzFKc1ZsZFZiR1JWWWtWc00xWXlOVWRXVjBwWlVXNUdWVlpzVmpSVmJGcExaRWRLUjFGc1pGTmlSbFkwVm10a01HRXhXWGxTYTFwUFYwVmFWbFpyV2t0VlJteHpWbTVrYWxac1NscFpNR1F3WVVaYVZWSnJiRmRTZWtJMFZrZDRZVkp0VGtoUFZsWlhWbXR3ZUZkc1ZtRmpNazVYVW14V1ZHRjZWbkJXYkZwWFRsWmFSMkZJWkdwTmEzQlhWR3hhVjFWc1drZFRiRVpYVFVkb2RsWlZXbGRqTVdSMFpFWkNVbFpFUVRWYWExcFhVMWRLTmxWdGVGTldSbHBoVjJ0b2QxWXhVWGxXYms1VFYwaENWbFpzV2t0Uk1YQkZVbTVPV0ZKc2NGcFhhMXBEVmpGS2RHUjZSbGROYm1oVVZqSnplRkl5U2tkaFJsWnBWa1ZhVVZkc1pEQldhekZIVlc1U1RsWXdXbTlVVlZKVFRsWmFSMkZIT1ZwV2Eyd3pWbXhTUjFaWFJuSmpSVGxoVmxad00xa3hXa3RYVjBwR1QxZDRhVll5YURWV2ExcFhZVEZrZEZac1drNVdSbkJXVmpCa05GVkdVbGhsUldSUFZteEdORlpYZEhkVWJFcFZWbXRhVjJKSGFIcFdNbk40WTJ4a2RWVnNWbGRXYTNCVVZrZDRZV1F4VGtkWGJHeHBVbnBHVkZacldtRlRSbHAwWTBVNVRrMVZTbE5WUmxGNFQwVXhSVlJVUmxCU01XdDNWR3BLV21Rd05WVlRWRUpQVWpBd2VWZHJaRk5pVlRWSVZHMXNVRlpIZERaVWJGSlNaREF4Y1ZGVVRsQldSMUp3VkdwS1lXRkZNVVZSVkZaaFVrZDRjVmRYY0hOaFZUbFZWVzB4WVZJd05YUlVXSEJXWlZVeFJWWlVTbHBXUjAwd1ZERmtUazB4Y0hWbFJUbFRWbTFSTkdaRVFtbFBSMDVzVFZkS2JVNTZVbTFQVjBwcFRYcHJkMWxxWXpST1JFRXhXbFJOZWs1cVJteE5WMHByVFdwTk1rNUVaM3BPTWxac1dtcGpNRTFFVFhwTmFrVjZXWHBWTWs1cWJHdGFhbFY1V2tScmQxbHFiRGhVYTFaWVprRTlQUT09fFVIbzVOVTVIYUZGVFYzUnFVbFJDYmxGVU1XWk5lbU14V0hrd2VHWkhWWGhhUkVVMVdsUlJNVmxVUVROYVJFVXlUakpSTUU5VVNYbGFhbVJzVGtSRk5FNUVaM2RPVkdSb1RXcENiVnBFU1RSUFYxWnRXVEpHYVZsNlFYcE5hbEY1VFVSQmVWbDZWVFZOUjFKcVQxUm9hMDFVYkRoVWExWllaa0U5UFE9PXxOb25lfDZiYzE1NGEwN2UwOTc0ZDgyYjhiZDYxNDBjM2VkNjYxNzA3NzJjMjZiMzY0MjQ3YThhZjBkNGY2Zjk0NzQ0N2V8TkVXfA"

        page = 1
        while True:
            # 传入 json中返回的 bookmarks 去进行翻页
            list_content = pinterest_util.get_list(key_word, bookmarks)
            # 解析list   返回  data 和   bookmarks
            datas,bookmarks=pinterest_util.parse_list(list_content)
            page+=1
            data_mon = {'_id':str(key_word)+'_'+str(page),
                        'page':page,
                        'key_word':key_word,
                        'bookmarks':bookmarks,
                        }
            # insert_mon_data(tablename='pinterest_crawler_info',insert_data=data_mon)
            try:
                MongoDBUtil.insert_one('pinterest_crawler_info', data_mon)
            except Exception as e:
                print(e)
                print(key_word,page,'已经存在，跳过')
                continue
            # 判断是否有data
            if datas:
                # 去拿单个页的数据
                for result in datas:
                    pinterest_images = mongo_manager("pinterest_images", db="car_images")
                    # 请求 获取单个页的数据
                    id = result['id']
                    if id in donlist:
                        continue
                    else:
                        try:
                            insert_data = {'_id': id,'id': id,'key_word':key_word}
                            MongoDBUtil.insert_one('pinterest_images', insert_data)
                        except Exception as e:
                            print(e)
                    path = f'pinterest/{key_word}/{id}/'
                    if os.path.exists(path):
                        name = os.listdir(path)
                        if len(name) > 2:
                            print('数据已存在')
                            insert_data = {'_id': id, 'id': id, 'key_word': key_word, 'status': 'success'}
                            try:
                                MongoDBUtil.insert_one('pinterest_images', insert_data)
                            except Exception as e:
                                pinterest_images.updateOne({'_id': insert_data['_id']}, insert_data)
                                print(e)
                            continue
                    else:
                        os.mkdir(path)
                    content = pinterest_util.get_content_detail(id)
                #     解析单个页的数据 变成结构化的数据并保存到本地
                    if content is None:
                        print('no get imageurl data')
                        continue
                    data = pinterest_util.parse_content_detail(content)
                    if data:
                        # data  =[src,title,description,tag_list]

                        filename = str(data[0]).split('/')[-1]
                        result=down_image(file=str(path)+str(filename),url=data[0])
                        if result == 0:
                            if os.path.exists(path):
                                shutil.rmtree(path)
                            continue
                        txt_filename = filename.split('.jpg')[0] +'.txt'
                        txt_data = str(data[0])+','+str(data[2])
                        for tag in data[3]:
                            txt_data += ','+str(tag)
                        save_txt(PWD=path,txt_filename=txt_filename, data=txt_data)
                        images=[{
                            "img_file": filename,
                            "label_file": txt_filename,
                            "tag": data[3],
                            "title": data[1],
                            "description": data[2]
                        }]
                        jsondata = {"page_url": f'https://www.pinterest.com/pin/{id}/', "images": images}
                        save_summary(path=path, file='summary.json', jsondata=jsondata)
                        # 保存爬取信息到达mongodb
                        #data = [src, title, description, tag_list]
                        insert_data = {'_id': id, 'id': id, 'key_word': key_word, 'src': data[0], 'title': data[1],
                                       'description': data[2],
                                       'tag_list': data[3],'status':'success'}
                        # insert_mon_data(tablename='pinterest_images',insert_data=insert_data)
                        pinterest_images.updateOne({'_id': insert_data['_id']}, insert_data)
                    pinterest_images.close()
            else:
                break




def down_image(file, url):      # 下载图片方法
    print("开始下载：", url)
    for i in range(5):
        try:
            response = requests.get(url,proxies=proxies,timeout=10)
            with open(file, 'wb') as fd:
                fd.write(response.content)
            requests.session().close()
            return 1
        except Exception as e:
            time.sleep(5)
    print("下载失败了",url)
    return 0


def save_summary(path, file, jsondata):
    with open(path + file, 'w') as f:
        f.write(json.dumps(jsondata))
    f.close()


def save_txt(PWD,txt_filename, data):
    with open(PWD + txt_filename, 'w') as f:
        f.write(data)
    f.close()



if __name__ == '__main__':
    donlist=[]
    pinterest_images = mongo_manager("pinterest_images", db="car_images")
    seeds = pinterest_images.findAll({'status':'success'})
    for seed in seeds:
        donlist.append(seed['id'])
        # print(seed['id'])
        # seed['status'] = 'success'
        pinterest_images.updateOne({'_id': seed['_id']}, seed)
    print(len(donlist))
    pinterest_images.close()

    keys = []
    for ss in key_words:
        for k, v in ss.items():
            keys.append(v.strip())
    print(keys)
    keys1 =['concept car','futuristic car',
            'Automotive','super car',
            'fantastic car', 'amazing car',
            'cool car', 'fabulous car',
            'splendid car', 'brilliant car',
            'wonderful car', 'fascinating car',
            'colorful car', 'beautiful car',
            ]
    print(keys1)
    keys +=keys1
    pc = pinterest_crawler(event_size=len(keys))
    pc.init_seed(keys)
    pc.start()
    # #
    #
    # # test
    pass




